#!/user/bin/python
# _*_ coding: UTF-8 _*_
# Filename: goods.py

# 打开浏览器，自动化操作
# 安装python模块，selenium 操作浏览器驱动的模块  数据解密，js加密
import json
from time import ctime
import requests
from selenium import webdriver
import time
import pymongo
from bson.objectid import ObjectId
from selenium.webdriver.chrome.options import Options

client = pymongo.MongoClient('mongodb://localhost:27017')
mydb = client["jdmpresystem"]
product = mydb['products']

# 搜索商品
from selenium.common.exceptions import NoSuchElementException



# 页面下拉
def drop_down():
    for x in range(1, 10, 2):
        j = x / 9
        js = 'document.documentElement.scrollTop = document.documentElement.scrollHeight * %f' % j
        driver.execute_script(js)
        time.sleep(0.5)


def yeshu():
    pages = driver.find_element_by_xpath('//*[@id="J_bottomPage"]/span[2]/em[1]/b').get_attribute('innerHTML')
    print(pages)
    for i in range(0, int(pages)):
        # for i in range(0, 1):
        parse_data()


# 不排除的店
nobrand = [
    '21克手机旗舰店',
    '百乐事手机官方旗舰店',
    '酷比旗舰店',
    'a axon_us海外官方旗舰店',
    "索野官方旗舰店",
    "克里特KRETA手机官方旗舰店",
    'CONQUEST手机旗舰店',
    '遨游旗舰店',
    "尼凯恩手机旗舰店",
    '康佳手机旗舰店',
    '小黄蜂旗舰店',
    '欧奇手机官方旗舰店',
    '爱户外手机官方旗舰店',
    'GEMRY手机旗舰店', ]


# 解析页面，获取详情页的url地址和初步判断是否是官方店铺
def parse_data():
    drop_down()
    lis = driver.find_elements_by_css_selector('#J_goodsList > ul > li')
    for li in lis:
        shopid = li.find_element_by_css_selector(' div > div.p-shop').get_attribute('innerHTML')
        shopid = shopid.replace('\n', '').strip()
        if len(shopid) == 0:
            continue
        else:
            shopname = li.find_element_by_css_selector(' div > div.p-shop > span > a') \
                .get_attribute('innerHTML')
            isself = shopname.find('京东自营')
            isself2 = shopname.find('京东官方自营')
            if isself == -1 & isself2 == -1:
                for bname in nobrand:
                    if shopname != bname:
                        continue
                    else:
                        shopnamearr.append(shopname)
                        price = li.find_element_by_class_name('p-price').get_attribute('innerHTML')
                        price = price.split('>')[4].split('<')[0].split('.')[0]
                        if price.isdigit():
                            pricearr.append(int(price))
                        else:
                            break
                        href.append(li.find_element_by_css_selector('.p-name a').get_attribute('href'))
                        break
                continue
            else:
                shopnamearr.append(shopname)
                price = li.find_element_by_class_name('p-price').get_attribute('innerHTML')
                price = price.split('>')[4].split('<')[0].split('.')[0]
                if price.isdigit():
                    pricearr.append(int(price))
                else:
                    pricearr.append(0)
                href.append(li.find_element_by_css_selector('.p-name a').get_attribute('href'))
    driver.find_element_by_xpath('//*[@id="J_bottomPage"]/span[1]/a[9]').click()


# 不需要爬的店名
notname = ['四川联通京东自营旗舰店', '上海联通京东自营旗舰店', '云南联通京东自营旗舰店', '安徽电信京东自营旗舰店', '小米移动京东自营旗舰店',
           '山东电信京东自营旗舰店', '天翼电信京东自营旗舰店', '中国移动终端京东自营旗舰店', '河南移动京东自营旗舰店', '上海电信京东自营旗舰店',
           '四川电信京东自营旗舰店', '四川移动京东自营旗舰店', '联想手机京东自营官方旗舰店', '成都联通京东自营旗舰店', '重庆联通京东自营旗舰店',
           '云南电信京东自营旗舰店', '中国联通京东自营旗舰店', '江苏移动京东自营店', '广东联通京东自营旗舰店', '广州电信京东自营旗舰店',
           '北京移动京东自营旗舰店', '中国联通京东自营官方旗舰店', '深圳移动京东自营旗舰店', '重庆电信京东自营旗舰店', '上海移动京东官方自营旗舰店'
                                                                         '陕西电信京东自营旗舰店']


# 详情页面判断是否爬取数据
def isGetData(hrefurl, price, shopname):
    driver.get(hrefurl)
    ptype = driver.execute_script('return pageConfig.product.pType')
    if ptype == 2:
        return
    else:
        flag = 0
        for sname in notname:
            isname = sname.find(shopname)
            if isname != -1:
                flag = 1
                break
            else:
                flag = 0
        if flag == 1:
            return
        else:
            jixing = driver.find_element_by_xpath('//*[@id="crumb-wrap"]/div/div[1]/div[9]').text
            hasData = product.find_one({'jixing': jixing})
            if hasData is None:
                get_data(shopname, jixing, price)
            else:
                print('数据已存在')
                return


# 取得视频介绍和图片
def getImageAndVideo(vid):
    imageAndVideo = []
    response = requests.get('https://cd.jd.com/tencent/video_v2?callback=jQuery267636&vid=' + vid)
    url = response.text
    vurl = url.split('playUrl')[1]
    vurl = vurl.split('"')[2]
    imgurl = url.split('imageUrl')[1]
    imgurl = imgurl.split('"')[2]
    imageAndVideo.append(vurl)
    imageAndVideo.append(imgurl)
    return imageAndVideo


# 爬取参数
def get_data(shopname, jixing, price):
    skuid = driver.execute_script('return pageConfig.product.skuid')
    colorSize = driver.execute_script('return pageConfig.product.colorSize')
    Comment = [{}]
    if len(colorSize) != 0:
        for i in range(0, len(colorSize)):
            sku = colorSize[i]['skuId']
            response = requests.get('https://p.3.cn/prices/mgets?skuIds=' + str(sku))
            colorSize[i]['price'] = int(response.json()[0]['p'].split('.')[0])
    # 得到宣传图片和视频
    imageAndVideo = []
    imageAndVideoJson = driver.execute_script('return pageConfig.product.imageAndVideoJson'),
    for i in imageAndVideoJson:
        if i != {}:
            if 'infoVideoId' in i:
                if 'mainVideoId' in i:
                    vid = i['mainVideoId']
                    imageAndVideo = getImageAndVideo(vid)
                else:
                    vid = i['infoVideoId']
                    imageAndVideo = getImageAndVideo(vid)
            else:
                if 'mainVideoId' in i:
                    vid = i['mainVideoId']
                    imageAndVideo = getImageAndVideo(vid)
                else:
                    imageAndVideo = []
    # 预售
    isYuShou = driver.execute_script('return pageConfig.product.isYuShou')
    if isYuShou is None:
        isYuShou = False
    else:
        isYuShou = True
    params = {
        "skuid": skuid,
        'brand': driver.find_element_by_xpath('//*[@id="crumb-wrap"]/div/div[1]/div[7]/div/div/div[1]/a').get_attribute(
            'innerHTML'),
        'price': price,
        'shopname': shopname,
        'jixing': jixing,
        'isYuShou': isYuShou,
        'skuname': driver.execute_script('return pageConfig.product.name'),
        'colorSize': colorSize,
        'imageAndVideo': imageAndVideo,
        'imageList': driver.execute_script('return pageConfig.product.imageList'),
        'product_info': [],
        'cpu':'',
        'rearcamera':'',
        'system':'',
        'proRam': '',
        'proRom': '',
        'ptable': [],
        'screen': 0,
        'ostime': 0,
        'comment': Comment,
        'commentCounts': '',
        'commentcontent': [],
        'clickTimes': 0
    }
    product_infos = driver.find_elements_by_xpath('//*[@id="detail"]/div[2]/div[1]/div[1]/ul[3]/li')
    for p in product_infos:
        phtml = p.get_attribute('innerHTML')
        if 'CPU型号' in phtml:
            params['cpu'] = phtml.split('：')[1]
        if '系统' in phtml:
            params['system'] = phtml.split('：')[1]
        if '运行内存' in phtml:
            nowram = phtml.split('：')[1]
            if nowram == '2GB以下':
                params['proRam'] = 1
            else:
                nowram = "".join(list(filter(str.isdigit, nowram)))
                if nowram.isdigit() == False:
                    nowram = 0
                else:
                    nowram = int(nowram)
                params['proRam'] = nowram
        if '机身内存' in phtml:
            nowrom = phtml.split('：')[1]
            if nowrom == '8GB以下':
                params['proRom'] = 7
            else:
                nowrom = "".join(list(filter(str.isdigit, nowrom)))
                if nowrom.isdigit() == False:
                    nowrom = 0
                else:
                    nowrom = int(nowrom)
            params['proRom'] = nowrom
        params['product_info'].append(phtml)

    ptables = driver.find_elements_by_xpath('//*[@id="detail"]/div[2]/div[2]/div[1]/div')
    osyear = ''
    osmonth = ''
    for pt in ptables:
        dic = {}
        keys = pt.find_element_by_tag_name('h3').get_attribute("innerHTML")
        dic[keys] = {}
        pdd = pt.find_elements_by_css_selector('dl > dl')
        for pd in pdd:
            keys2 = pd.find_element_by_css_selector('dt').get_attribute('innerHTML')
            dic[keys][keys2] = {}
            dd = pd.find_elements_by_css_selector('dd')
            if (len(dd) == 1):
                value = pd.find_element_by_css_selector('dd').get_attribute('innerHTML')
            else:
                value = pd.find_element_by_css_selector('dd:nth-child(odd)').get_attribute('innerHTML')
            if keys2 == '上市月份':
                osmonth = "".join(list(filter(str.isdigit, value)))
                if len(osmonth) == 1:
                    osmonth = '0' + osmonth
            if keys2 == '上市年份':
                osyear = "".join(list(filter(str.isdigit, value)))
            if keys2 == '主屏幕尺寸':
                screen = "".join(list(filter(str.isdigit, value)))
                if screen.isdigit() == False:
                    screen = 0
                else:
                    screen = int(screen)
                    if screen > 1000:
                        screen = screen / 1000
                    elif screen > 100:
                        screen = screen / 100
                    elif screen > 10:
                        screen = screen / 10
                    else:
                        screen = screen
                params['screen'] = screen
            dic[keys][keys2] = value
        if osyear + osmonth != '':
            params['ostime'] = int(osyear + osmonth)
        else:
            params['ostime'] = 0
        params['ptable'].append(dic)
    # https://club.jd.com/comment/productCommentSummaries.action?referenceIds=100020961264
    params['packItems'] = driver.find_element_by_xpath('//*[@id="detail"]/div[2]/div[2]/div[2]/p').get_attribute(
        'innerHTML')
    driver.find_element_by_xpath('//*[@id="detail"]/div[1]/ul/li[5]').click()
    commentcont = driver.find_elements_by_class_name('tag-1')
    for ct in commentcont:
        params['commentcontent'].append(ct.get_attribute('innerHTML'))

    # 爬取数据
    Commenturl = 'https://club.jd.com/comment/productCommentSummaries.action?referenceIds=' + str(skuid)
    driver.get(Commenturl)
    com = driver.find_element_by_css_selector('body').get_attribute('innerHTML')
    com = json.loads(com)
    Comment[0]['CommentCountStr'] = com['CommentsCount'][0]['CommentCountStr']
    counts = com['CommentsCount'][0]['CommentCountStr']
    if '万' in counts:
        counts = "".join(list(filter(str.isdigit, counts)))
        params['commentCounts'] = int(counts) * 10000
    else:
        counts = "".join(list(filter(str.isdigit, counts)))
        params['commentCounts'] = int(counts)
    Comment[0]['GoodRateShow'] = com['CommentsCount'][0]['GoodRateShow']
    Comment[0]['GoodCountStr'] = com['CommentsCount'][0]['GoodCountStr']
    Comment[0]['VideoCountStr'] = com['CommentsCount'][0]['VideoCountStr']
    Comment[0]['GeneralCountStr'] = com['CommentsCount'][0]['GeneralCountStr']
    Comment[0]['PoorCountStr'] = com['CommentsCount'][0]['PoorCountStr']
    Comment[0]['AfterCountStr'] = com['CommentsCount'][0]['AfterCountStr']
    params['comment'] = Comment

    x = product.insert_one(params)
    print(x)


# 分割href 防止没有数据
def arr_size(arr, size):
    s = []
    for i in range(0, int(len(arr)), size):
        c = arr[i:i + size]
        s.append(c)
    return s



if __name__ == '__main__':
    print('%start', ctime())

    keyword = "手机"
    # 1. 创建一个浏览器对象
    chrome_options = Options()
    chrome_options.add_argument('--headless')
    chrome_options.add_argument('--disable-gpu')
    driver = webdriver.Chrome(options=chrome_options)
    url = "https://www.jd.com"
    driver.get(url)
    # time.sleep(2)
    # driver.delete_all_cookies()
    # # 2. 执行页面操作
    # # 加载 cookies信息
    # with open("cookies.txt", "r") as f:
    #     cookies = json.load(f)
    #     for cookie in cookies:
    #         driver.add_cookie(cookie)
    #
    # # 验证是否登录成功
    # driver.get(url)
    driver.implicitly_wait(10)  # 设置浏览器的等待（耗时） 智能化等待 加载完就不会等待
    driver.maximize_window()
    driver.find_element_by_css_selector('#key').send_keys(keyword)
    driver.find_element_by_css_selector('#search > div > div.form > button').click()
    href = []
    pricearr = []
    shopnamearr = []
    yeshu()
    plen = 0
    for detailurl in href:
        isGetData(detailurl, pricearr[plen], shopnamearr[plen])
        plen = plen + 1
    print('end%s', ctime())
